package org.correct.crawler;

import java.io.FileNotFoundException;
import java.io.PrintWriter;
import java.net.URI;
import java.net.URISyntaxException;
import java.util.ArrayList;
import java.util.Calendar;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import org.correct.config.Config;
import org.correct.model.HtmlItem;
import org.correct.util.Counter;
import org.correct.util.PageInfoAccessor;

public class AnalyzeManager2 {
	
	private class SuffixChecker {
		private String[] allowedSuffix = {
			"html", "jsp", "asp", "aspx", "php", "htm"	
		};
		
		public boolean isAllowed(String suffix) {
			for(int i = 0; i < allowedSuffix.length; i++)
				if(suffix.equalsIgnoreCase(allowedSuffix[i]))
					return true;
			return false;
		}
	}
	
	private StoreManager storeManager;
	private CrawlManager2 crawlManager;
	private Counter counter;
	private final int maxQueueLength;
	private HashMap<String, HtmlItem> urlTable;
	private ArrayList<Filter> filterList;
	private DenyDomainFilter denyDomainFilter;
	private final int minIndexLength = 1000;
	private SuffixChecker suffixChecker = new SuffixChecker();
	private boolean anyLimit = false;
	private int maxNumHtml;
	
	public AnalyzeManager2(String fileBase, 
			int maxStoreThreadNum, int maxQueueLength, 
			String allowDomain, String denyDomain) {
		storeManager = new StoreManager(fileBase, maxStoreThreadNum);
		counter = new Counter();
		this.maxQueueLength = maxQueueLength;
		urlTable = new HashMap<String, HtmlItem>();
		filterList = new ArrayList<Filter>();
		filterList.add(new AllowDomainFilter(allowDomain));
		denyDomainFilter = new DenyDomainFilter(denyDomain);
		filterList.add(denyDomainFilter);
	}
	
	public AnalyzeManager2(String fileBase, 
			int maxStoreThreadNum, int maxQueueLength, 
			String allowDomain, String[] denyDomain) {
		storeManager = new StoreManager(fileBase, maxStoreThreadNum);
		counter = new Counter();
		this.maxQueueLength = maxQueueLength;
		urlTable = new HashMap<String, HtmlItem>();
		filterList = new ArrayList<Filter>();
		filterList.add(new AllowDomainFilter(allowDomain));
		denyDomainFilter = new DenyDomainFilter(denyDomain);
		filterList.add(denyDomainFilter);
	}
	
	public void setHtmlNumLimit(int num) {
		this.maxNumHtml = num;
		this.anyLimit = true;
	}
	
	public void setCrawlManager(CrawlManager2 crawlManager) {
		this.crawlManager = crawlManager;
	}
	
	public void start() {
		storeManager.start();
	}
	
	/*
	public void addHtml(String url, String html, String encoding, Calendar time) {
		// System.out.println("-> " + Thread.currentThread() + " in addHtml()");
		
		if(html.length() < minIndexLength) {
			denyDomainFilter.addDenyDomain(url);
			return;
		}
		
		// System.out.println("-> " + Thread.currentThread() + " after check");
		int qLength = storeManager.queueLength();
		System.out.println("Storage Queue Length: " + qLength);
		if(qLength >= maxQueueLength)
			return;
		
		synchronized(urlTable) {
			if(urlTable.containsKey(url))
				return;
		}
		
		int id = counter.inc();
		HtmlItem htmlItem = new HtmlItem(id, html, encoding, time);
		HtmlItem htmlItemForCheck = new HtmlItem(id, encoding, time);
		
		synchronized(urlTable) {
			urlTable.put(url, htmlItemForCheck);
		}
		
		storeManager.addFile(htmlItem);
		
		if(id >= this.maxNumHtml && anyLimit == true)
			synchronized(this) {
				writePageInfoTemp();
				System.exit(0);
			}
		extract(url, id, html);
	}
	*/
	
	public void addHtml(String url, String html, String encoding, Calendar time) {
		// System.out.println("-> " + Thread.currentThread() + " in addHtml()");
		
		if(html.length() < minIndexLength)
			return;
		
		// System.out.println("-> " + Thread.currentThread() + " after check");
		int qLength = storeManager.queueLength();
		System.out.println("Storage Queue Length: " + qLength);
		if(qLength >= maxQueueLength)
			return;
		
		int id = counter.inc();
		HtmlItem htmlItem = new HtmlItem(id, html, encoding, time);
		HtmlItem urlItem = null; 
		
		synchronized(urlTable) {
			urlItem = urlTable.get(url);
			urlItem.id = id;
			urlItem.encoding = encoding;
			urlItem.time = time;
		}
		
		storeManager.addFile(htmlItem);
		
		if(id >= this.maxNumHtml && anyLimit == true)
			synchronized(this) {
				writePageInfoTemp();
				storeManager.shutDown();
				System.exit(0);
			}
		extract(url, id, html);
	}
	
	private void writePageInfoTemp() {
		try {
			PrintWriter w = new PrintWriter(Config.PAGE_INFO_PATH);
			String url = null;
			HtmlItem html = null;
			synchronized(urlTable) {
				for(Map.Entry<String, HtmlItem> entry : urlTable.entrySet()) {
					url = entry.getKey();
					html = entry.getValue();
					if(html.id == -1)
						continue;
					
					w.println(html.id + PageInfoAccessor.separater + 
							url + PageInfoAccessor.separater + 
							html.encoding + PageInfoAccessor.separater + 
							html.time.getTimeInMillis());
				}
			}
			w.flush();
			w.close();
		} catch (FileNotFoundException e) {
			e.printStackTrace();
		}
	}

	public void addDenyDomain(String url) {
		denyDomainFilter.addDenyDomain(url);
	}
	
	private void extract(String url, int id, String html) {
		System.out.println("Extracting file #" + id + " ...");
		if(html == null)
			return;
		
		Pattern pattern = Pattern.compile("<[aA] href ?= ?\"(.+?)\"");
		Matcher matcher = pattern.matcher(html);
		String nextUrl = null;
		
		Pattern pSuffix = Pattern.compile(".+\\.([a-zA-Z]+)$");
		Matcher mSuffix = null;
		String suffix = null;
		
		int iSharp = 0;
		int iSlash = 0;
		URI tempUri = null;
		
		while(matcher.find()) {
			nextUrl = matcher.group(1);
			
			// trim anchors
			iSharp = nextUrl.lastIndexOf("#");
			if(iSharp >= 0)
				nextUrl = nextUrl.substring(0, nextUrl.lastIndexOf("#"));
			
			// check document suffix
			mSuffix = pSuffix.matcher(nextUrl);
			if(mSuffix.find()) {
				suffix = mSuffix.group(1);
				if(!suffixChecker.isAllowed(suffix))
					continue;
			}

			// trace relative URL's
			if(!(nextUrl.startsWith("http://") || nextUrl.startsWith("https://"))) {
				iSlash = url.lastIndexOf("/");
				if(iSlash > 0)
					nextUrl = url.substring(0, url.lastIndexOf("/")) +
						(nextUrl.startsWith("/") ? nextUrl : ("/" + nextUrl));
				else
					nextUrl = url + (nextUrl.startsWith("/") ? nextUrl : ("/" + nextUrl));
			}
			
			try {
				tempUri = new URI(nextUrl);
				nextUrl = tempUri.normalize().toASCIIString();
				if(tempUri.getPort() != -1)
					continue;
			} catch (URISyntaxException e) {
				e.printStackTrace();
				continue;
			}
			
			if(allowFetch(nextUrl) && tryAddUrl(nextUrl)) {
				crawlManager.addUrl(nextUrl);
			}
		}
	}
	
	public boolean tryAddUrl(String url) {
		synchronized(urlTable) {
			if(urlTable.containsKey(url))
				return false;
			urlTable.put(url, new HtmlItem());
		}
		return true;
	}
	
	public boolean noDuplicateUrl(String url) {
		synchronized(urlTable) {
			if(urlTable.containsKey(url))
				return false;
		}
		return true;
	}
	
	private boolean allowFetch(String url) {
		for(Filter f : filterList) {
			if(f.filter(url))
				return false;
		}
		
		return true;
	}
}
